
# The following packages will be needed in the project
import os 
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sbn
''' magic word %matplotlib inline' is used so that our visualizations are plotted inline with 
#he notebook. See this page for more: http://ipython.readthedocs.io/en/stable/interactive/magics.html
'''
%matplotlib inline
import requests
import tweepy 
import json 
from timeit import default_timer as timer


#read in the downloaded "twitter_archive_enhanced.csv" into a data from the computer
Df_twitter_archive =pd.read_csv("twitter-archive-enhanced.csv")


# save the image-predictions.tsv file and load it into Df_image_predictions   
Df_image_predictions = pd.read_csv('image-predictions.tsv',sep="\t")


#reading the line by line to get each json data and store in a list

with open('tweet_json.txt', 'r') as json_file:
    Tweet_json_list = []
    for line in json_file:
        lines = line.replace('\n', '')
        Tweet_json_list.append(lines)
'''
extract 'tweet_id' ,'retweet_count', and'favourite_count' from tweet_json_list and store in 
Df_twitterApi_data
https://stackoverflow.com/questions/67560603/loading-tweet-json-file-with-multiple-json-objects-into-pandas-dataframe
'''
line_length = len(Tweet_json_list)
tweet_id = []
retweet_count = []
favourite_count = []
tweet_date = []
data = {'tweet_id': [], 'retweet_count':[], 'favorite_count':[]}
for i in range(line_length):
    dictionary = json.loads(Tweet_json_list[i])
    tweet_id.append(dictionary['id'])
    retweet_count.append(dictionary['retweet_count'])
    favourite_count.append(dictionary['favorite_count'])
data.update({'tweet_id':tweet_id, 'retweet_count': retweet_count, 'favorite_count' : favourite_count})
Df_twitterApi_data = pd.DataFrame(data)


# Set Pandas column width and rows to allow longer text displays inorder to investigate better.
pd.set_option("display.max_colwidth",150)
pd.options.display.max_rows = 3000
#View dataframe
Df_twitter_archive


#look at the geneal properties of the data
Df_twitter_archive.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2356 entries, 0 to 2355
Data columns (total 17 columns):
 #   Column                      Non-Null Count  Dtype  
---  ------                      --------------  -----  
 0   tweet_id                    2356 non-null   int64  
 1   in_reply_to_status_id       78 non-null     float64
 2   in_reply_to_user_id         78 non-null     float64
 3   timestamp                   2356 non-null   object 
 4   source                      2356 non-null   object 
 5   text                        2356 non-null   object 
 6   retweeted_status_id         181 non-null    float64
 7   retweeted_status_user_id    181 non-null    float64
 8   retweeted_status_timestamp  181 non-null    object 
 9   expanded_urls               2297 non-null   object 
 10  rating_numerator            2356 non-null   int64  
 11  rating_denominator          2356 non-null   int64  
 12  name                        2356 non-null   object 
 13  doggo                       2356 non-null   object 
 14  floofer                     2356 non-null   object 
 15  pupper                      2356 non-null   object 
 16  puppo                       2356 non-null   object 
dtypes: float64(4), int64(3), object(10)
memory usage: 313.0+ KB


# Check the archive for duplicate records
print("There are {} duplicates records in the data" .format(Df_twitter_archive.duplicated().sum()))

There are 0 duplicates records in the data


Df_twitter_archive.isnull().sum()

tweet_id                         0
in_reply_to_status_id         2278
in_reply_to_user_id           2278
timestamp                        0
source                           0
text                             0
retweeted_status_id           2175
retweeted_status_user_id      2175
retweeted_status_timestamp    2175
expanded_urls                   59
rating_numerator                 0
rating_denominator               0
name                             0
doggo                            0
floofer                          0
pupper                           0
puppo                            0
dtype: int64


# Examine the name column further especially names with 4 characters or less
Df_twitter_archive.name[Df_twitter_archive.name.apply(lambda x: len(x)<=4)].unique()

array(['None', 'Jax', 'Zoey', 'Koda', 'Ted', 'Jim', 'Zeke', 'such',
       'Maya', 'Earl', 'Lola', 'Yogi', 'Noah', 'Gus', 'Alfy', 'Koko',
       'Rey', 'Gary', 'a', 'Jack', 'Emmy', 'Beau', 'Aja', 'Cash', 'Coco',
       'Jed', 'Kody', 'Dawn', 'Cody', 'Lili', 'Dave', 'Burt', 'Carl',
       'Thor', 'Luna', 'Arya', 'Iggy', 'Kyle', 'Leo', 'Odin', 'Tuck',
       'Hank', 'Ken', 'Max', 'Odie', 'Arlo', 'Lucy', 'Ava', 'Rory', 'Eli',
       'Ash', 'Tobi', 'not', 'Kuyu', 'Pete', 'Kyro', 'Loki', 'Mia', 'one',
       'Mutt', 'Bear', 'Kona', 'Phil', 'Ike', 'Mo', 'Toby', 'Nala',
       'Gabe', 'Luca', 'Finn', 'Anna', 'Bo', 'Tom', 'Dido', 'Levi', 'Alf',
       'Sky', 'Tyr', 'Mary', 'Moe', 'Halo', 'Sam', 'Ito', 'Milo', 'Cali',
       'Duke', 'Chef', 'Doc', 'Sobe', 'Iroh', 'Ruby', 'Mack', 'Juno',
       'Lily', 'Newt', 'Nida', 'BeBe', 'mad', 'Dale', 'Hero', 'Godi',
       'Dash', 'Bell', 'Jay', 'Mya', 'an', 'Huck', 'very', 'O', 'Blue',
       'Fizz', 'Chip', 'Grey', 'Al', 'just', 'Lou', 'Tito', 'Brat',
       'Tove', 'my', 'Kota', 'Eve', 'Rose', 'Theo', 'Fido', 'Emma',
       'Gert', 'Dex', 'Ace', 'Fred', 'Zoe', 'Blu', 'his', 'Cora', 'Abby',
       'Geno', 'Beya', 'Kilo', 'Doug', 'Aqua', 'Axel', 'Remy', 'this',
       'Ziva', 'Puff', 'all', 'Ivar', 'Sid', 'Otis', 'Suki', 'Ebby',
       'Link', 'Ozzy', 'old', 'Zeus', 'Nico', 'Siba', 'Kanu', 'Opie',
       'Kane', 'Sora', 'Lacy', 'Olaf', 'Kara', 'Zara', 'Bode', 'Rudy',
       'Fiji', 'Rilo', 'Yoda', 'Chet', 'Kaia', 'Eazy', 'CeCe', 'Ole',
       'Berb', 'Bob', 'Kobe', 'Lolo', 'Eriq', 'the', 'Durg', 'Fynn',
       'Ferg', 'Trip', 'Brad', 'Opal', 'Marq', 'Mona', 'Birf', 'Oreo',
       'Jeph', 'Obi', 'Tino', 'Lupe', 'Lulu', 'Taco', 'Joey', 'Kreg',
       'Todo', 'Tess', 'by', 'Mike', 'Evy', 'Tug', 'Izzy', 'Chuq', 'Karl',
       'Herm', 'Bert', 'Zuzu', 'Jeb', 'life', 'Acro', 'Obie', 'Dot',
       'Mac', 'Ed', 'Taz', 'Jazz', 'Rolf', 'Cal', 'Tuco', 'Mojo', 'Mark',
       'JD', 'Pip', 'Jett', 'Amy', 'Sage', 'Andy', 'Creg', 'Gin', 'Bloo',
       'Edd', 'Herb', 'Liam', 'Ben', 'Skye', 'Dug', 'Kirk', 'Ralf',
       'Chaz', 'Bobb', 'Hanz', 'Zeek', 'Maks', 'Jo', 'DayZ', 'Ron',
       'Erik', 'Stu', 'Kial', 'Dook', 'Hall', 'Fwed', 'Keet'],
      dtype=object)


# set pattern to match dog names in lower cases
lowercase_pattern= r"[a-z].?"
wrong=Df_twitter_archive.name.str.match(lowercase_pattern, ).sum()
print("There are {} improper names in lowercase!".format(wrong))

There are 109 improper names in lowercase!


# check text with names in lowercase to comfirm whether there are dogs names or not.
lower_df=Df_twitter_archive[Df_twitter_archive.name.str.match(lowercase_pattern)]
lower_df[['text','name']]


#check to see if there are any lowercase names that are not dogs names
'''function to determine names that are not dogs'''
def Not_dogs(text_column):
  wrong_dogs= text_column.str.contains('We only rate dogs' or 'only send dogs' or 'stop sending sharks').sum()
  print("There are {} none dogs rating names!".format(wrong_dogs))
Not_dogs(lower_df['text'])

There are 21 none dogs rating names!


#check if there are other names that are not dogs in the data frame
Not_dogs(Df_twitter_archive['text'])

There are 56 none dogs rating names!


Df_twitter_archive.describe()


# Further investigate denominator equal 0.by view the text column
Df_twitter_archive[Df_twitter_archive['rating_denominator'] ==0].text

313    @jonnysun @Lin_Manuel ok jomny I know you're excited but 960/00 isn't a valid rating, 13/10 is tho
Name: text, dtype: object


# Print the data on the screen
Df_image_predictions


#look at the genral properties of the data
Df_image_predictions.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2075 entries, 0 to 2074
Data columns (total 12 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   tweet_id  2075 non-null   int64  
 1   jpg_url   2075 non-null   object 
 2   img_num   2075 non-null   int64  
 3   p1        2075 non-null   object 
 4   p1_conf   2075 non-null   float64
 5   p1_dog    2075 non-null   bool   
 6   p2        2075 non-null   object 
 7   p2_conf   2075 non-null   float64
 8   p2_dog    2075 non-null   bool   
 9   p3        2075 non-null   object 
 10  p3_conf   2075 non-null   float64
 11  p3_dog    2075 non-null   bool   
dtypes: bool(3), float64(3), int64(2), object(4)
memory usage: 152.1+ KB


Df_image_predictions.duplicated().sum()

0


Df_twitter_archive.expanded_urls.isnull().sum()

59


# visual assessment
Df_twitterApi_data


Df_twitterApi_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2326 entries, 0 to 2325
Data columns (total 3 columns):
 #   Column          Non-Null Count  Dtype
---  ------          --------------  -----
 0   tweet_id        2326 non-null   int64
 1   retweet_count   2326 non-null   int64
 2   favorite_count  2326 non-null   int64
dtypes: int64(3)
memory usage: 54.6 KB


# Make copies of original pieces of data
df_archive_clean = Df_twitter_archive.copy()
df_predictions_clean = Df_image_predictions.copy()
df_twitterApi_clean = Df_twitterApi_data.copy()


#create a filter for original tweets
filter_value = (df_archive_clean['in_reply_to_status_id'].isna()) & \
                 (df_archive_clean['retweeted_status_id'].isna())

#Filter rows with only original tweets
df_archive_clean = df_archive_clean[filter_value]

#Drop redundant retweets and replys columns.
column_drop =['in_reply_to_status_id','in_reply_to_user_id','retweeted_status_id','retweeted_status_user_id',
'retweeted_status_timestamp']
df_archive_clean.drop(column_drop, inplace=True, axis=1 )


df_archive_clean.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 2097 entries, 0 to 2355
Data columns (total 12 columns):
 #   Column              Non-Null Count  Dtype 
---  ------              --------------  ----- 
 0   tweet_id            2097 non-null   int64 
 1   timestamp           2097 non-null   object
 2   source              2097 non-null   object
 3   text                2097 non-null   object
 4   expanded_urls       2094 non-null   object
 5   rating_numerator    2097 non-null   int64 
 6   rating_denominator  2097 non-null   int64 
 7   name                2097 non-null   object
 8   doggo               2097 non-null   object
 9   floofer             2097 non-null   object
 10  pupper              2097 non-null   object
 11  puppo               2097 non-null   object
dtypes: int64(3), object(9)
memory usage: 213.0+ KB


# Convert tweet_id to string/object data type for te three datasets
df_archive_clean['tweet_id'] = df_archive_clean.tweet_id.astype(str)
df_predictions_clean['tweet_id']=df_predictions_clean.tweet_id.astype(str)
df_twitterApi_clean.tweet_id = df_twitterApi_clean.tweet_id.astype(str)
# Convert timestamp to a pandas datetime object
df_archive_clean['timestamp'] = pd.to_datetime(df_archive_clean['timestamp'])


print(df_archive_clean[['tweet_id','timestamp']].dtypes)
print(df_twitterApi_clean.tweet_id.info())
print(df_predictions_clean.tweet_id.info())

tweet_id                  object
timestamp    datetime64[ns, UTC]
dtype: object
<class 'pandas.core.series.Series'>
RangeIndex: 2326 entries, 0 to 2325
Series name: tweet_id
Non-Null Count  Dtype 
--------------  ----- 
2326 non-null   object
dtypes: object(1)
memory usage: 18.3+ KB
None
<class 'pandas.core.series.Series'>
RangeIndex: 2075 entries, 0 to 2074
Series name: tweet_id
Non-Null Count  Dtype 
--------------  ----- 
2075 non-null   object
dtypes: object(1)
memory usage: 16.3+ KB
None


'''Extract tweet source from source column and save in tweet_source column
and Convert it to categorical type'''
pattern = r'\>(.*)\<'
df_archive_clean['tweet_source'] = df_archive_clean.source.str.extract(pattern).astype('category')
# Drop source column
df_archive_clean.drop(columns="source", inplace= True)


df_archive_clean['tweet_source'].value_counts()

Twitter for iPhone     1964
Vine - Make a Scene      91
Twitter Web Client       31
TweetDeck                11
Name: tweet_source, dtype: int64


df_archive_clean.columns

Index(['tweet_id', 'timestamp', 'text', 'expanded_urls', 'rating_numerator',
       'rating_denominator', 'name', 'doggo', 'floofer', 'pupper', 'puppo',
       'tweet_source'],
      dtype='object')


#create a filter for improper names that are not dogs#
wrong_dogs= df_archive_clean.text.str.contains('We only rate dogs' or 'only send dogs' or 'stop sending sharks')
'''filter out names that are not dogs names from the dataset.'''
df_archive_clean = df_archive_clean[~wrong_dogs]


lowercase_filter=df_archive_clean.name.str.match(r"[a-z].?")
lowercase_total=lowercase_filter.sum()
#Check the entire data for names in lower cases
print("There are {} inappropriate names remaining!".format(lowercase_total))

There are 86 inappropriate names remaining!


##filter the remaining improper names
names_left = df_archive_clean.name[lowercase_filter].unique()
#Replace them with 'None'
for i in names_left:
    df_archive_clean['name'].replace(i, 'None', inplace = True)


assert df_archive_clean['name'].str.match(r"[a-z].?").sum() == 0


#search for rating_denominator whose value is 10
df_archive_clean[df_archive_clean['rating_denominator'] ==0].index

Int64Index([], dtype='int64')


#comfirm that ther is no such value in rating_denominator
df_archive_clean[df_archive_clean['rating_denominator'] ==0].rating_denominator.any()

False


# Drop the expanded urls from df_archive_clean
df_archive_clean.drop(columns='expanded_urls', inplace = True)


df_archive_clean.columns

Index(['tweet_id', 'timestamp', 'text', 'rating_numerator',
       'rating_denominator', 'name', 'doggo', 'floofer', 'pupper', 'puppo',
       'tweet_source'],
      dtype='object')


# Extract tweets text from text column and save into a tweets_text column
df_archive_clean['tweets_text'] = df_archive_clean['text'].str.replace(r"(http.+)", '', regex=True)

# Extract urls into a tweet url column
df_archive_clean['tweet_url'] = df_archive_clean['text'].str.extract(r"(http.+)")

# Fill records where tweet text does not contain a url with None
df_archive_clean['tweet_url'].fillna('None', inplace = True)
#Drop 'text' column
df_archive_clean.drop(columns="text", inplace= True)


df_archive_clean[['tweet_url','tweets_text']].sample(5)


df_archive_clean.columns

Index(['tweet_id', 'timestamp', 'rating_numerator', 'rating_denominator',
       'name', 'doggo', 'floofer', 'pupper', 'puppo', 'tweet_source',
       'tweets_text', 'tweet_url'],
      dtype='object')


# create empty lists to save the best value from each row in the dataset
dog_breed_list = []

# create a function that will iterates through prediction columns to find the best prediction and confidence probability
def breed_confidence(row):
    """
        row: means a row from our dataframe of interest.        
    """
    if row.p1_dog:
        dog_breed_list.append({'dog_breed': row.p1,'confidence_probability': row.p1_conf})
    elif row.p2_dog:
        dog_breed_list.append({'dog_breed': row.p2,'confidence_probability': row.p2_conf})
    elif row.p3_dog:
        dog_breed_list.append({'dog_breed': row.p3,'confidence_probability': row.p3_conf})
    else:
        dog_breed_list.append({'dog_breed': 'Unknown breed','confidence_probability': 0})
        
 # Apply our breed_confidence function on our df_master_clean
df_predictions_clean[['p1','p2','p3', 'p1_dog','p2_dog','p2_conf','p3_dog','p1_conf','p2_conf','p3_conf',]].apply(breed_confidence, axis=1)

# Assign the values in dog_breed_list into new columns in our df_master_clean
df_predictions_clean[['dog_breed', 'confidence_probabilty']] = pd.DataFrame( dog_breed_list)
#drop the colums that have been colapsed into two new columns
df_predictions_clean.drop(columns=['p1', 'p1_conf', 'p1_dog', 'p2','p2_conf', 'p2_dog', 'p3', 'p3_conf', 'p3_dog'], inplace=True)


df_predictions_clean.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2075 entries, 0 to 2074
Data columns (total 5 columns):
 #   Column                 Non-Null Count  Dtype 
---  ------                 --------------  ----- 
 0   tweet_id               2075 non-null   object
 1   jpg_url                2075 non-null   object
 2   img_num                2075 non-null   int64 
 3   dog_breed              2075 non-null   object
 4   confidence_probabilty  2075 non-null   object
dtypes: int64(1), object(4)
memory usage: 81.2+ KB


# Remove all underscores and format the dog_breed names and convert to titlecase.
df_predictions_clean.dog_breed = df_predictions_clean.dog_breed.str.replace('_', ' ').str.title()


df_predictions_clean.dog_breed.str.contains('_','-').value_counts()

False    2075
Name: dog_breed, dtype: int64


df_predictions_clean.dog_breed.sample(10)

1892                  Unknown Breed
1598                       Malamute
122                            Chow
1844                  Unknown Breed
1102                        Samoyed
692     Soft-Coated Wheaten Terrier
609                   Unknown Breed
1349             Labrador Retriever
452                          Vizsla
1937                  Unknown Breed
Name: dog_breed, dtype: object


#rename columns to have more discriptivenames
df_archive_clean.rename(columns={'timestamp':'tweet_date', 'name':'dog_name'}, inplace=True)
df_predictions_clean.rename(columns={'jpg_url':'image_url', 'img_num':'image_number'}, inplace=True)
df_twitterApi_clean.rename(columns={'retweet_count': 'retweets', 'favorite_count':'likes'}, inplace=True)


print('column in df_achive= ',df_archive_clean.columns)
print('column in df_predictions= ',df_predictions_clean.columns)
print('column in df_twitterApi= ',df_twitterApi_clean.columns)

column in df_achive=  Index(['tweet_id', 'tweet_date', 'rating_numerator', 'rating_denominator',
       'dog_name', 'doggo', 'floofer', 'pupper', 'puppo', 'tweet_source',
       'tweets_text', 'tweet_url'],
      dtype='object')
column in df_predictions=  Index(['tweet_id', 'image_url', 'image_number', 'dog_breed',
       'confidence_probabilty'],
      dtype='object')
column in df_twitterApi=  Index(['tweet_id', 'retweets', 'likes'], dtype='object')


# Make copies of original pieces of cleaned data
df_archive = df_archive_clean.copy()
df_predictions = df_predictions_clean.copy()
df_twitterApi = df_twitterApi_clean.copy()


# reset dataframe indices for df_twitter and predictions_clean
df_predictions=df_predictions.reset_index(drop=True)
df_twitterApi= df_twitterApi.reset_index(drop=True)


# Merge archive clean and prediction clean into master df
df_master = pd.merge(df_archive, df_predictions,on='tweet_id', how='inner' )

# Merge json clean into master df
df_master_clean = pd.merge(df_master, df_twitterApi, on='tweet_id', how='inner')


# Check results 
df_master_clean.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 1915 entries, 0 to 1914
Data columns (total 18 columns):
 #   Column                 Non-Null Count  Dtype              
---  ------                 --------------  -----              
 0   tweet_id               1915 non-null   object             
 1   tweet_date             1915 non-null   datetime64[ns, UTC]
 2   rating_numerator       1915 non-null   int64              
 3   rating_denominator     1915 non-null   int64              
 4   dog_name               1915 non-null   object             
 5   doggo                  1915 non-null   object             
 6   floofer                1915 non-null   object             
 7   pupper                 1915 non-null   object             
 8   puppo                  1915 non-null   object             
 9   tweet_source           1915 non-null   object             
 10  tweets_text            1915 non-null   object             
 11  tweet_url              1915 non-null   object             
 12  image_url              1915 non-null   object             
 13  image_number           1915 non-null   int64              
 14  dog_breed              1915 non-null   object             
 15  confidence_probabilty  1915 non-null   object             
 16  retweets               1915 non-null   int64              
 17  likes                  1915 non-null   int64              
dtypes: datetime64[ns, UTC](1), int64(5), object(12)
memory usage: 284.3+ KB


#Here is how to clean them:

# Replace none values in columns diggo, floofer, pupper,and puppo with nothing
df_master_clean.doggo.replace('None', '', inplace=True)
df_master_clean.floofer.replace('None', '', inplace=True)
df_master_clean.pupper.replace('None', '', inplace=True)
df_master_clean.puppo.replace('None', '', inplace=True)

# merge diggo, floofer, pupper,and puppo into column 'dog_state'
df_master_clean['dog_stage'] = df_master_clean.doggo + df_master_clean.floofer + df_master_clean.pupper + df_master_clean.puppo


df_master_clean['dog_stage'].value_counts()

                1613
pupper           201
doggo             62
puppo             22
doggopupper        8
floofer            7
doggopuppo         1
doggofloofer       1
Name: dog_stage, dtype: int64


# Identify and isolate records where dogs were assigned multiple stages
multiple_stage_filter = df_master_clean.dog_stage.apply(lambda x: x in ['doggopupper', 'doggopuppo', 'doggofloofer'])

# Examine these occurences
multstage_df=df_master_clean[multiple_stage_filter][['tweet_id', 'image_url', 'tweets_text', 'dog_stage']]
multstage_df


# Remove the records with ids '686749460672679938',& '699088579889332224'
filter = df_master_clean['tweet_id'].isin(['686749460672679938', '699088579889332224'])
#drop the filter rows
df_master_clean=df_master_clean[~filter]

# Correct the erroneously categorized records
correct_dict = { 
    '674793399141146624':'pupper',
    '678643457146150913':'pupper',
    '684940049151070208':'pupper',
    '669680153564442624':'floofer',
    '669564461267722241':'doggo',
    '705428427625635840':'pupper',
    '676821958043033607':'doggo',
    '715342466308784130':'pupper',
    '724405726123311104': 'pupper'
}

for key in correct_dict.keys():
    df_master_clean.loc[df_master_clean['tweet_id'] == key, 'dog_stage'] = correct_dict[key]


# Check if the filter record has been dropped
df_master_clean[filter]

C:\Users\USER\AppData\Local\Temp\ipykernel_8772\2618290396.py:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  df_master_clean[filter]


# handle missing values
df_master_clean.loc[df_master_clean.dog_stage == '', 'dog_stage'] = np.nan


#change datatype of dog_stage to category
df_master_clean['dog_stage']=df_master_clean.dog_stage.astype('category')

# change dog stage category to tittle case.
df_master_clean.dog_stage=df_master_clean.dog_stage.apply(lambda x: x.title())
#drop columns not needed
df_master_clean.drop(columns=['doggo','floofer','pupper','puppo'], inplace=True)


df_master_clean.dog_stage.value_counts()

Pupper          204
Doggo            64
Puppo            22
Doggopupper       8
Floofer           8
Doggofloofer      1
Doggopuppo        1
Name: dog_stage, dtype: int64


# Verify the datatype in the stage column
df_master_clean.dtypes

tweet_id                              object
tweet_date               datetime64[ns, UTC]
rating_numerator                       int64
rating_denominator                     int64
dog_name                              object
tweet_source                          object
tweets_text                           object
tweet_url                             object
image_url                             object
image_number                           int64
dog_breed                             object
confidence_probabilty                 object
retweets                               int64
likes                                  int64
dog_stage                           category
dtype: object


# save cleaned master dataset to CSV
df_master_clean.to_csv('twitter_archive_master.csv', index = False)


# read our twitter_archive_master.csv into data frame df_master
df_master_archive = pd.read_csv('twitter_archive_master.csv')


df_master_archive.sample(3)


#filter out dog_breed ='Unknown Breed' and dog_stage='Not specified'
selected_breeds =df_master_archive.loc[(df_master_archive.dog_breed !='Unknown Breed')].reset_index(drop=True)
#1. What is the most popular dog breed interns of total likes
Top10_most_popular_breeds = selected_breeds.groupby('dog_breed').sum()['likes'].nlargest(10)
'''convert top10 object to data frame using the pd.DataFrame and the zip functions'''
df_Top10_most_popular_breeds=pd.DataFrame(list(zip(Top10_most_popular_breeds.index,Top10_most_popular_breeds.values)),columns=['Dog breed','Total likes'])
df_Top10_most_popular_breeds


#create for folder to store our visuals
if not os.path.exists('visuals'):
    os.makedirs('visuals')
#Bar plot for Total Likes Per Predicted Dog Breed
ax = sbn.barplot(x = "Dog breed", y = "Total likes", data = df_Top10_most_popular_breeds,
                      palette = 'hls')
ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha='right')
ax.set_title("Total Likes Per Predicted Dog Breed")
'''show only the left and bottom axis line'''
sbn.despine(left = False, bottom = False)


#filter for only known dog_state
specified_stage=df_master_archive.query("dog_stage!='null' ").reset_index(drop=True)
#aggregate by dog_stage
groups=specified_stage.groupby('dog_stage').agg({'likes': "mean",'retweets':"mean"})
top_stage=groups.nlargest(n=5, columns=['likes','retweets']).round()
print(top_stage);

                likes  retweets
dog_stage                      
Doggopuppo    41809.0   16076.0
Puppo         19504.0    5302.0
Doggo         17312.0    5974.0
Doggofloofer  14797.0    2784.0
Doggopupper   12670.0    3708.0


# create stacked bar chart for dog_state
top_stage.plot(kind='bar', stacked=False, color=['c', 'pink'])
 # Add Title and Labels
plt.title('Dog Stage per percentage likes and retweets')
plt.xlabel('Dog Stage')
plt.ylabel('Average liks and retweets')
sbn.despine(left = False, bottom = False)


#sellect the likes and retweets and find the correlation fators
df_relationship = df_master_archive[['likes', 'retweets']]
corr = df_relationship.corr()
print(corr)

             likes  retweets
likes     1.000000  0.927829
retweets  0.927829  1.000000


#plot the correlation matrix for likes and retweets
sbn.heatmap(corr, annot=True, linewidths=.4, fmt= '.1f')
plt.title('Correlation Matrix')

Text(0.5, 1.0, 'Correlation Matrix')


# Plot scatter plot for the likes and retweets
an=sbn.scatterplot(x="likes",
                    y="retweets",
                    palette = 'hls',
                    data=df_master_archive)
'''set the title and the position of the legend'''                                   
tit=an.set_title(" Relation between retweets and Likes from tweet_source")
plt.setp(tit, color='c') 
sbn.despine(left = False, bottom = False)
plt.savefig('visuals/fig4_scatterplot.png')


pd.set_option("display.max_colwidth",50)
df_master_archive.sample(2).T

`

	text	name
22	I've yet to rate a Venezuelan Hover Wiener. This is such an honor. 14/10 paw-inspiring af (IG: roxy.thedoxy) https://t.co/20VrLAA8ba	such
56	Here is a pupper approaching maximum borkdrive. Zooming at never before seen speeds. 14/10 paw-inspiring af \n(IG: puffie_the_chow) https://t.co/g...	a
118	RT @dog_rates: We only rate dogs. This is quite clearly a smol broken polar bear. We'd appreciate if you only send dogs. Thank you... 12/10…	quite
169	We only rate dogs. This is quite clearly a smol broken polar bear. We'd appreciate if you only send dogs. Thank you... 12/10 https://t.co/g2nSyGenG9	quite
193	Guys, we only rate dogs. This is quite clearly a bulbasaur. Please only send dogs. Thank you... 12/10 human used pet, it's super effective https:/...	quite
335	There's going to be a dog terminal at JFK Airport. This is not a drill. 10/10 \nhttps://t.co/dp5h9bCwU7	not
369	Occasionally, we're sent fantastic stories. This is one of them. 14/10 for Grace https://t.co/bZ4axuH6OK	one
542	We only rate dogs. Please stop sending in non-canines like this Freudian Poof Lion. This is incredibly frustrating... 11/10 https://t.co/IZidSrBvhi	incredibly
649	Here is a perfect example of someone who has their priorities in order. 13/10 for both owner and Forrest https://t.co/LRyMrU7Wfq	a
682	RT @dog_rates: Say hello to mad pupper. You know what you did. 13/10 would pet until no longer furustrated https://t.co/u1ulQ5heLX	mad
759	RT @dog_rates: This is an East African Chalupa Seal. We only rate dogs. Please only send in dogs. Thank you... 10/10 https://t.co/iHe6liLwWR	an
773	RT @dog_rates: We only rate dogs. Pls stop sending in non-canines like this Mongolian grass snake. This is very frustrating. 11/10 https://…	very
801	Guys this is getting so out of hand. We only rate dogs. This is a Galapagos Speed Panda. Pls only send dogs... 10/10 https://t.co/8lpAGaZRFn	a
819	We only rate dogs. Pls stop sending in non-canines like this Arctic Floof Kangaroo. This is very frustrating. 11/10 https://t.co/qlUDuPoE3d	very
822	RT @dog_rates: This is just downright precious af. 12/10 for both pupper and doggo https://t.co/o5J479bZUC	just
852	This is my dog. Her name is Zoey. She knows I've been rating other dogs. She's not happy. 13/10 no bias at all https://t.co/ep1NkYoiwB	my
924	This is one of the most inspirational stories I've ever come across. I have no words. 14/10 for both doggo and owner https://t.co/I5ld3eKD5k	one
988	What jokester sent in a pic without a dog in it? This is not @rock_rates. This is @dog_rates. Thank you ...10/10 https://t.co/nDPaYHrtNX	not
992	That is Quizno. This is his beach. He does not tolerate human shenanigans on his beach. 10/10 reclaim ur land doggo https://t.co/vdr7DaRSa7	his
993	This is one of the most reckless puppers I've ever seen. How she got a license in the first place is beyond me. 6/10 https://t.co/z5bAdtn9kd	one
1002	This is a mighty rare blue-tailed hammer sherk. Human almost lost a limb trying to take these. Be careful guys. 8/10 https://t.co/TGenMeXreW	a
1004	Viewer discretion is advised. This is a terrible attack in progress. Not even in water (tragic af). 4/10 bad sherk https://t.co/L3U0j14N5R	a
1017	This is a carrot. We only rate dogs. Please only send in dogs. You all really should know this by now ...11/10 https://t.co/9e48aPrBm2	a
1025	This is an Iraqi Speed Kangaroo. It is not a dog. Please only send in dogs. I'm very angry with all of you ...9/10 https://t.co/5qpBTTpgUt	an
1031	We only rate dogs. Pls stop sending in non-canines like this Jamaican Flop Seal. This is very very frustrating. 9/10 https://t.co/nc53zEN0hZ	very
1040	This is actually a pupper and I'd pet it so well. 12/10\nhttps://t.co/RNqS7C4Y4N	actually
1049	This is a very rare Great Alaskan Bush Pupper. Hard to stumble upon without spooking. 12/10 would pet passionately https://t.co/xOBKCdpzaa	a
1063	This is just downright precious af. 12/10 for both pupper and doggo https://t.co/o5J479bZUC	just
1071	This is getting incredibly frustrating. This is a Mexican Golden Beaver. We only rate dogs. Only send dogs ...10/10 https://t.co/0yolOOyD3X	getting
1095	Say hello to mad pupper. You know what you did. 13/10 would pet until no longer furustrated https://t.co/u1ulQ5heLX	mad
1097	We only rate dogs. Please stop sending in non-canines like this Alaskan Flop Turtle. This is very frustrating. 10/10 https://t.co/qXteK6Atxc	very
1120	Say hello to this unbelievably well behaved squad of doggos. 204/170 would try to pet all at once https://t.co/yGQI3He3xv	this
1121	We only rate dogs. Pls stop sending non-canines like this Bulgarian Eyeless Porch Bear. This is unacceptable... 9/10 https://t.co/2yctWAUZ3Z	unacceptable
1138	This is all I want in my life. 12/10 for super sleepy pupper https://t.co/4RlLA5ObMh	all
1193	People please. This is a Deadly Mediterranean Plop T-Rex. We only rate dogs. Only send in dogs. Thanks you... 11/10 https://t.co/2ATDsgHD4n	a
1206	This is old now but it's absolutely heckin fantastic and I can't not share it with you all. 13/10 https://t.co/wJX74TSgzP	old
1207	This is a taco. We only rate dogs. Please only send in dogs. Dogs are what we rate. Not tacos. Thank you... 10/10 https://t.co/cxl6xGY8B9	a
1259	We 👏🏻 only 👏🏻 rate 👏🏻 dogs. Pls stop sending in non-canines like this Dutch Panda Worm. This is infuriating. 11/10 https://t.co/odfLzBonG2	infuriating
1340	Here is a heartbreaking scene of an incredible pupper being laid to rest. 10/10 RIP pupper https://t.co/81mvJ0rGRu	a
1351	Here is a whole flock of puppers. 60/50 I'll take the lot https://t.co/9dpcw6MdWa	a
1361	This is a Butternut Cumberfloof. It's not windy they just look like that. 11/10 back at it again with the red socks https://t.co/hMjzhdUHaW	a
1362	This is an East African Chalupa Seal. We only rate dogs. Please only send in dogs. Thank you... 10/10 https://t.co/iHe6liLwWR	an
1368	This is a Wild Tuscan Poofwiggle. Careful not to startle. Rare tongue slip. One eye magical. 12/10 would def pet https://t.co/4EnShAQjv6	a
1382	"Pupper is a present to world. Here is a bow for pupper." 12/10 precious as hell https://t.co/ItSsE92gCW	a
1385	We only rate dogs. Pls stop sending in non-canines like this Mongolian grass snake. This is very frustrating. 11/10 https://t.co/22x9SbCYCU	very
1435	Please stop sending in saber-toothed tigers. This is getting ridiculous. We only rate dogs.\n...8/10 https://t.co/iAeQNueou8	getting
1457	This is just a beautiful pupper good shit evolution. 12/10 https://t.co/2L8pI0Z2Ib	just
1499	This is a rare Arctic Wubberfloof. Unamused by the happenings. No longer has the appetites. 12/10 would totally hug https://t.co/krvbacIX0N	a
1527	Stop sending in lobsters. This is the final warning. We only rate dogs. Thank you... 9/10 https://t.co/B9ZXXKJYNx	the
1603	This is the newly formed pupper a capella group. They're just starting out but I see tons of potential. 8/10 for all https://t.co/wbAcvFoNtn	the
1693	This is actually a lion. We only rate dogs. For the last time please only send dogs. Thank u.\n12/10 would still pet https://t.co/Pp26dMQxap	actually
1724	This is by far the most coordinated series of pictures I was sent. Downright impressive in every way. 12/10 for all https://t.co/etzLo3sdZE	by
1737	Guys this really needs to stop. We've been over this way too many times. This is a giraffe. We only rate dogs.. 7/10 https://t.co/yavgkHYPOC	a
1747	This is officially the greatest yawn of all time. 12/10 https://t.co/4R0Cc0sLVE	officially
1785	This is a dog swinging. I really enjoyed it so I hope you all do as well. 11/10 https://t.co/Ozo9KHTRND	a
1797	This is the happiest pupper I've ever seen. 10/10 would trade lives with https://t.co/ep8ATEJwRb	the
1815	This is the saddest/sweetest/best picture I've been sent. 12/10 😢🐶 https://t.co/vQ2Lw1BLBF	the
1853	This is a Sizzlin Menorah spaniel from Brooklyn named Wylie. Lovable eyes. Chiller as hell. 10/10 and I'm out.. poof https://t.co/7E0AiJXPmI	a
1854	Seriously guys?! Only send in dogs. I only rate dogs. This is a baby black bear... 11/10 https://t.co/H7kpabTfLj	a
1877	C'mon guys. We've been over this. We only rate dogs. This is a cow. Please only submit dogs. Thank you...... 9/10 https://t.co/WjcELNEqN2	a
1878	This is a fluffy albino Bacardi Columbia mix. Excellent at the tweets. 11/10 would hug gently https://t.co/diboDRUuEI	a
1916	This is life-changing. 12/10 https://t.co/SroTpI6psB	life
1923	This is a Sagitariot Baklava mix. Loves her new hat. 11/10 radiant pup https://t.co/Bko5kFJYUU	a
1936	This is one esteemed pupper. Just graduated college. 10/10 what a champ https://t.co/nyReCVRiyd	one
1941	This is a heavily opinionated dog. Loves walls. Nobody knows how the hair works. Always ready for a kiss. 4/10 https://t.co/dFiaKZ9cDl	a
1955	This is a Lofted Aphrodisiac Terrier named Kip. Big fan of bed n breakfasts. Fits perfectly. 10/10 would pet firmly https://t.co/gKlLpNzIl3	a
1994	This is a baby Rand Paul. Curls for days. 11/10 would cuddle the hell out of https://t.co/xHXNaPAYRe	a
2001	This is light saber pup. Ready to fight off evil with light saber. 10/10 true hero https://t.co/LPPa3btIIt	light
2019	This is just impressive I have nothing else to say. 11/10 https://t.co/LquQZiZjJP	just
2030	This is space pup. He's very confused. Tries to moonwalk at one point. Super spiffy uniform. 13/10 I love space pup https://t.co/SfPQ2KeLdq	space
2034	This is a Tuscaloosa Alcatraz named Jacob (Yacōb). Loves to sit in swing. Stellar tongue. 11/10 look at his feet https://t.co/2IslQ8ZSc7	a
2037	This is the best thing I've ever seen so spread it like wildfire & maybe we'll find the genius who created it. 13/10 https://t.co/q6RsuOVYwU	the
2066	This is a Helvetica Listerine named Rufus. This time Rufus will be ready for the UPS guy. He'll never expect it 9/10 https://t.co/34OhVhMkVr	a
2116	This is a Deciduous Trimester mix named Spork. Only 1 ear works. No seat belt. Incredibly reckless. 9/10 still cute https://t.co/CtuJoLHiDo	a
2125	This is a Rich Mahogany Seltzer named Cherokee. Just got destroyed by a snowball. Isn't very happy about it. 9/10 https://t.co/98ZBi6o4dj	a
2128	This is a Speckled Cauliflower Yosemite named Hemry. He's terrified of intruder dog. Not one bit comfortable. 9/10 https://t.co/yV3Qgjh8iN	a
2146	This is a spotted Lipitor Rumpelstiltskin named Alphred. He can't wait for the Turkey. 10/10 would pet really well https://t.co/6GUGO7azNX	a
2153	This is a brave dog. Excellent free climber. Trying to get closer to God. Not very loyal though. Doesn't bark. 5/10 https://t.co/ODnILTr4QM	a
2161	This is a Coriander Baton Rouge named Alfredo. Loves to cuddle with smaller well-dressed dog. 10/10 would hug lots https://t.co/eCRdwouKCl	a
2191	This is a Slovakian Helter Skelter Feta named Leroi. Likes to skip on roofs. Good traction. Much balance. 10/10 wow! https://t.co/Dmy2mY2Qj5	a
2198	This is a wild Toblerone from Papua New Guinea. Mouth always open. Addicted to hay. Acts blind. 7/10 handsome dog https://t.co/IGmVbz07tZ	a
2204	This is an Irish Rigatoni terrier named Berta. Completely made of rope. No eyes. Quite large. Loves to dance. 10/10 https://t.co/EM5fDykrJg	an
2211	Here is a horned dog. Much grace. Can jump over moons (dam!). Paws not soft. Bad at barking. 7/10 can still pet tho https://t.co/2Su7gmsnZm	a
2212	Never forget this vine. You will not stop watching for at least 15 minutes. This is the second coveted.. 13/10 https://t.co/roqIxCvEB3	the
2218	This is a Birmingham Quagmire named Chuk. Loves to relax and watch the game while sippin on that iced mocha. 10/10 https://t.co/HvNg9JWxFt	a
2222	Here is a mother dog caring for her pups. Snazzy red mohawk. Doesn't wag tail. Pups look confused. Overall 4/10 https://t.co/YOHe6lf09m	a
2235	This is a Trans Siberian Kellogg named Alfonso. Huge ass eyeballs. Actually Dobby from Harry Potter. 7/10 https://t.co/XpseHBlAAb	a
2249	This is a Shotokon Macadamia mix named Cheryl. Sophisticated af. Looks like a disappointed librarian. Shh (lol) 9/10 https://t.co/J4GnJ5Swba	a
2255	This is a rare Hungarian Pinot named Jessiga. She is either mid-stroke or got stuck in the washing machine. 8/10 https://t.co/ZU0i0KJyqD	a
2264	This is a southwest Coriander named Klint. Hat looks expensive. Still on house arrest :(\n9/10 https://t.co/IQTOMqDUIe	a
2273	This is a northern Wahoo named Kohl. He runs this town. Chases tumbleweeds. Draws gun wicked fast. 11/10 legendary https://t.co/J4vn2rOYFk	a
2287	This is a Dasani Kingfisher from Maine. His name is Daryl. Daryl doesn't like being swallowed by a panda. 8/10 https://t.co/jpaeu6LNmW	a
2304	This is a curly Ticonderoga named Pepe. No feet. Loves to jet ski. 11/10 would hug until forever https://t.co/cyDfaK8NBc	a
2311	This is a purebred Bacardi named Octaviath. Can shoot spaghetti out of mouth. 10/10 https://t.co/uEvsGLOFHa	a
2314	This is a golden Buckminsterfullerene named Johm. Drives trucks. Lumberjack (?). Enjoys wall. 8/10 would hug softly https://t.co/uQbZJM2DQB	a
2326	This is quite the dog. Gets really excited when not in water. Not very soft tho. Bad at fetch. Can't do tricks. 2/10 https://t.co/aMCTNWO94t	quite
2327	This is a southern Vesuvius bumblegruff. Can drive a truck (wow). Made friends with 5 other nifty dogs (neat). 7/10 https://t.co/LopTBkKa8h	a
2333	This is an extremely rare horned Parthenon. Not amused. Wears shoes. Overall very nice. 9/10 would pet aggressively https://t.co/QpRjllzWAL	an
2334	This is a funny dog. Weird toes. Won't come down. Loves branch. Refuses to eat his food. Hard to cuddle with. 3/10 https://t.co/IIXis0zta0	a
2335	This is an Albanian 3 1/2 legged Episcopalian. Loves well-polished hardwood flooring. Penis on the collar. 9/10 https://t.co/d9NcXFKwLv	an
2345	This is the happiest dog you will ever see. Very committed owner. Nice couch. 10/10 https://t.co/RhUEAloehK	the
2346	Here is the Rand Paul of retrievers folks! He's probably good at poker. Can drink beer (lol rad). 8/10 good dog https://t.co/pYAJkAe76p	the
2347	My oh my. This is a rare blond Canadian terrier on wheels. Only $8.98. Rather docile. 9/10 very rare https://t.co/yWBqbrzy8O	a
2348	Here is a Siberian heavily armored polar bear mix. Strong owner. 10/10 I would do unspeakable things to pet this dog https://t.co/rdivxLiqEt	a
2349	This is an odd dog. Hard on the outside but loving on the inside. Petting still fun. Doesn't play catch well. 2/10 https://t.co/v5A4vzSDdc	an
2350	This is a truly beautiful English Wilson Staff retriever. Has a nice phone. Privileged. 10/10 would trade lives with https://t.co/fvIbQfHjIe	a
2352	This is a purebred Piers Morgan. Loves to Netflix and chill. Always looks like he forgot to unplug the iron. 6/10 https://t.co/DWnyCjf2mx	a
2353	Here is a very happy pup. Big fan of well-maintained decks. Just look at that tongue. 9/10 would cuddle af https://t.co/y671yMhoiR	a
2354	This is a western brown Mitsubishi terrier. Upset about leaf. Actually 2 dogs here. 7/10 would walk the shit out of https://t.co/r7mOb2m0UI	a

	tweet_id	in_reply_to_status_id	in_reply_to_user_id	retweeted_status_id	retweeted_status_user_id	rating_numerator	rating_denominator
count	2.356000e+03	7.800000e+01	7.800000e+01	1.810000e+02	1.810000e+02	2356.000000	2356.000000
mean	7.427716e+17	7.455079e+17	2.014171e+16	7.720400e+17	1.241698e+16	13.126486	10.455433
std	6.856705e+16	7.582492e+16	1.252797e+17	6.236928e+16	9.599254e+16	45.876648	6.745237
min	6.660209e+17	6.658147e+17	1.185634e+07	6.661041e+17	7.832140e+05	0.000000	0.000000
25%	6.783989e+17	6.757419e+17	3.086374e+08	7.186315e+17	4.196984e+09	10.000000	10.000000
50%	7.196279e+17	7.038708e+17	4.196984e+09	7.804657e+17	4.196984e+09	11.000000	10.000000
75%	7.993373e+17	8.257804e+17	4.196984e+09	8.203146e+17	4.196984e+09	12.000000	10.000000
max	8.924206e+17	8.862664e+17	8.405479e+17	8.874740e+17	7.874618e+17	1776.000000	170.000000

	tweet_url	tweets_text
1814	https://t.co/mpQl2rJjDh	I promise this wasn't meant to be a cuteness overload account but ermergerd look at this cozy pupper. 13/10
2196	https://t.co/OmxouCZ8IY	Say hello to Bobb. Bobb is a Golden High Fescue & a proud father of 8. Bobb sleeps while the little pups play. 11/10
2120	https://t.co/1DJfHmfBQN	Meet Scooter. He's ready for his first day of middle school. Remarkable tongue. 12/10
1319	https://t.co/KUS0xl7XIp	Please enjoy this pup in a cooler. Permanently ready for someone to throw a tennis ball his way. 12/10
2207	https://t.co/3FuNbzyPwo	This is Timofy. He's a pilot for Southwest. It's Christmas morning & everyone has gotten kickass gifts but him. 9/10

	tweet_id	image_url	tweets_text	dog_stage
137	855851453814013952	https://pbs.twimg.com/media/C-CYWrvWAAU8AXH.jpg	Here's a puppo participating in the #ScienceMarch. Cleverly disguising her own doggo agenda. 13/10 would keep the planet habitable for	doggopuppo
143	854010172552949760	https://pbs.twimg.com/media/C9oNt91WAAAFSLS.jpg	At first I thought this was a shy doggo, but it's actually a Rare Canadian Floofer Owl. Amateurs would confuse the two. 11/10 only send dogs	doggofloofer
318	817777686764523521	https://pbs.twimg.com/ext_tw_video_thumb/817777588030476288/pu/img/KbuLpE4krHF4VdPf.jpg	This is Dido. She's playing the lead role in "Pupper Stops to Catch Snow Before Resuming Shadow Box with Dried Apple." 13/10 (IG: didodoggo)	doggopupper
373	808106460588765185	https://pbs.twimg.com/media/Czb4iFRXgAIUMiN.jpg	Here we have Burke (pupper) and Dexter (doggo). Pupper wants to be exactly like doggo. Both 12/10 would pet at same time	doggopupper
399	801115127852503040	https://pbs.twimg.com/media/Cx4h7zHUsAAqaJd.jpg	This is Bones. He's being haunted by another doggo of roughly the same size. 12/10 deep breaths pupper everything's fine	doggopupper
484	785639753186217984	https://pbs.twimg.com/media/CucnLmeWAAALOSC.jpg	This is Pinot. He's a sophisticated doggo. You can tell by the hat. Also pointier than your average pupper. Still 10/10 would pet cautiously	doggopupper
620	759793422261743616	https://pbs.twimg.com/media/CotUFZEWcAA2Pku.jpg	Meet Maggie & Lila. Maggie is the doggo, Lila is the pupper. They are sisters. Both 12/10 would pet at the same time	doggopupper
671	751583847268179968	https://pbs.twimg.com/media/Cm4phTpWcAAgLsr.jpg	Please stop sending it pictures that don't even have a doggo or pupper in them. Churlish af. 5/10 neat couch tho	doggopupper
758	741067306818797568	https://pbs.twimg.com/media/CkjMx99UoAM2B1a.jpg	This is just downright precious af. 12/10 for both pupper and doggo	doggopupper
799	733109485275860992	https://pbs.twimg.com/media/CiyHLocU4AI2pJu.jpg	Like father (doggo), like son (pupper). Both 12/10	doggopupper

	tweet_id	tweet_date	rating_numerator	rating_denominator	dog_name	tweet_source	tweets_text	tweet_url	image_url	image_number	dog_breed	confidence_probabilty	retweets	likes	dog_stage
529	776813020089548800	2016-09-16 16:00:31+00:00	12	10	Solomon	Twitter for iPhone	Meet Solomon. He was arrested for possession of adorable and attempted extra pats on the head. 12/10 would post bail	https://t.co/nFqLaOLUQA	https://pbs.twimg.com/media/CsfLUDbXEAAu0VF.jpg	1	Toy Poodle	0.51661	1110	4595	NaN
130	857989990357356544	2017-04-28 16:08:49+00:00	12	10	Rosie	Twitter for iPhone	This is Rosie. She was just informed of the walk that's about to happen. Knows there are many a stick along the way. 12/10 such excite	https://t.co/sOl7cFaP5X	https://pbs.twimg.com/media/C-gxV9ZXkAIBL-S.jpg	1	French Bulldog	0.43258	2259	14451	NaN
1700	670061506722140161	2015-11-27 02:08:07+00:00	11	10	Liam	Twitter for iPhone	This is Liam. He has a particular set of skills. He will look for you, he will find you, and he will kill you. 11/10	https://t.co/uQMFKv1vjn	https://pbs.twimg.com/media/CUyJYk1WoAMPROb.jpg	1	Italian Greyhound	0.329339	288	665	NaN

	Dog breed	Total likes
0	Golden Retriever	1654990
1	Labrador Retriever	953804
2	Pembroke	872347
3	Chihuahua	636368
4	French Bulldog	508627
5	Samoyed	391756
6	Cocker Spaniel	336872
7	Pug	316819
8	Chow	292524
9	Toy Poodle	273724

	1397	251
tweet_id	676776431406465024	829449946868879360
tweet_date	2015-12-15 14:50:49+00:00	2017-02-08 22:00:52+00:00
rating_numerator	10	11
rating_denominator	10	10
dog_name	None	None
tweet_source	Twitter for iPhone	Twitter for iPhone
tweets_text	When someone yells "cops!" at a party and you ...	Here's a stressed doggo. Had a long day. Many ...
tweet_url	https://t.co/4rMZi5Ca1k	https://t.co/fmRS43mWQB
image_url	https://pbs.twimg.com/ext_tw_video_thumb/67677...	https://pbs.twimg.com/media/C4LMUf8WYAkWz4I.jpg
image_number	1	1
dog_breed	Unknown Breed	Labrador Retriever
confidence_probabilty	0	0.315163
retweets	1796	1854
likes	4504	9846
dog_stage	NaN	Doggo

	tweet_id	jpg_url	img_num	p1	p1_conf	p1_dog	p2	p2_conf	p2_dog	p3	p3_conf	p3_dog
0	666020888022790149	https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg	1	Welsh_springer_spaniel	0.465074	True	collie	1.566650e-01	True	Shetland_sheepdog	6.142850e-02	True
1	666029285002620928	https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg	1	redbone	0.506826	True	miniature_pinscher	7.419170e-02	True	Rhodesian_ridgeback	7.201000e-02	True
2	666033412701032449	https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg	1	German_shepherd	0.596461	True	malinois	1.385840e-01	True	bloodhound	1.161970e-01	True
3	666044226329800704	https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg	1	Rhodesian_ridgeback	0.408143	True	redbone	3.606870e-01	True	miniature_pinscher	2.227520e-01	True
4	666049248165822465	https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg	1	miniature_pinscher	0.560311	True	Rottweiler	2.436820e-01	True	Doberman	1.546290e-01	True
5	666050758794694657	https://pbs.twimg.com/media/CT5Jof1WUAEuVxN.jpg	1	Bernese_mountain_dog	0.651137	True	English_springer	2.637880e-01	True	Greater_Swiss_Mountain_dog	1.619920e-02	True
6	666051853826850816	https://pbs.twimg.com/media/CT5KoJ1WoAAJash.jpg	1	box_turtle	0.933012	False	mud_turtle	4.588540e-02	False	terrapin	1.788530e-02	False
7	666055525042405380	https://pbs.twimg.com/media/CT5N9tpXIAAifs1.jpg	1	chow	0.692517	True	Tibetan_mastiff	5.827940e-02	True	fur_coat	5.444860e-02	False
8	666057090499244032	https://pbs.twimg.com/media/CT5PY90WoAAQGLo.jpg	1	shopping_cart	0.962465	False	shopping_basket	1.459380e-02	False	golden_retriever	7.958960e-03	True
9	666058600524156928	https://pbs.twimg.com/media/CT5Qw94XAAA_2dP.jpg	1	miniature_poodle	0.201493	True	komondor	1.923050e-01	True	soft-coated_wheaten_terrier	8.208610e-02	True
10	666063827256086533	https://pbs.twimg.com/media/CT5Vg_wXIAAXfnj.jpg	1	golden_retriever	0.775930	True	Tibetan_mastiff	9.371780e-02	True	Labrador_retriever	7.242660e-02	True
11	666071193221509120	https://pbs.twimg.com/media/CT5cN_3WEAAlOoZ.jpg	1	Gordon_setter	0.503672	True	Yorkshire_terrier	1.742010e-01	True	Pekinese	1.094540e-01	True
12	666073100786774016	https://pbs.twimg.com/media/CT5d9DZXAAALcwe.jpg	1	Walker_hound	0.260857	True	English_foxhound	1.753820e-01	True	Ibizan_hound	9.747050e-02	True
13	666082916733198337	https://pbs.twimg.com/media/CT5m4VGWEAAtKc8.jpg	1	pug	0.489814	True	bull_mastiff	4.047220e-01	True	French_bulldog	4.895950e-02	True
14	666094000022159362	https://pbs.twimg.com/media/CT5w9gUW4AAsBNN.jpg	1	bloodhound	0.195217	True	German_shepherd	7.825980e-02	True	malinois	7.562780e-02	True
15	666099513787052032	https://pbs.twimg.com/media/CT51-JJUEAA6hV8.jpg	1	Lhasa	0.582330	True	Shih-Tzu	1.661920e-01	True	Dandie_Dinmont	8.968830e-02	True
16	666102155909144576	https://pbs.twimg.com/media/CT54YGiWUAEZnoK.jpg	1	English_setter	0.298617	True	Newfoundland	1.498420e-01	True	borzoi	1.336490e-01	True
17	666104133288665088	https://pbs.twimg.com/media/CT56LSZWoAAlJj2.jpg	1	hen	0.965932	False	cock	3.391940e-02	False	partridge	5.206580e-05	False
18	666268910803644416	https://pbs.twimg.com/media/CT8QCd1WEAADXws.jpg	1	desktop_computer	0.086502	False	desk	8.554740e-02	False	bookcase	7.947970e-02	False
19	666273097616637952	https://pbs.twimg.com/media/CT8T1mtUwAA3aqm.jpg	1	Italian_greyhound	0.176053	True	toy_terrier	1.118840e-01	True	basenji	1.111520e-01	True
20	666287406224695296	https://pbs.twimg.com/media/CT8g3BpUEAAuFjg.jpg	1	Maltese_dog	0.857531	True	toy_poodle	6.306380e-02	True	miniature_poodle	2.558060e-02	True
21	666293911632134144	https://pbs.twimg.com/media/CT8mx7KW4AEQu8N.jpg	1	three-toed_sloth	0.914671	False	otter	1.525000e-02	False	great_grey_owl	1.320720e-02	False
22	666337882303524864	https://pbs.twimg.com/media/CT9OwFIWEAMuRje.jpg	1	ox	0.416669	False	Newfoundland	2.784070e-01	True	groenendael	1.026430e-01	True
23	666345417576210432	https://pbs.twimg.com/media/CT9Vn7PWoAA_ZCM.jpg	1	golden_retriever	0.858744	True	Chesapeake_Bay_retriever	5.478680e-02	True	Labrador_retriever	1.424090e-02	True
24	666353288456101888	https://pbs.twimg.com/media/CT9cx0tUEAAhNN_.jpg	1	malamute	0.336874	True	Siberian_husky	1.476550e-01	True	Eskimo_dog	9.341240e-02	True
25	666362758909284353	https://pbs.twimg.com/media/CT9lXGsUcAAyUFt.jpg	1	guinea_pig	0.996496	False	skunk	2.402450e-03	False	hamster	4.608630e-04	False
26	666373753744588802	https://pbs.twimg.com/media/CT9vZEYWUAAlZ05.jpg	1	soft-coated_wheaten_terrier	0.326467	True	Afghan_hound	2.595510e-01	True	briard	2.068030e-01	True
27	666396247373291520	https://pbs.twimg.com/media/CT-D2ZHWIAA3gK1.jpg	1	Chihuahua	0.978108	True	toy_terrier	9.396970e-03	True	papillon	4.576810e-03	True
28	666407126856765440	https://pbs.twimg.com/media/CT-NvwmW4AAugGZ.jpg	1	black-and-tan_coonhound	0.529139	True	bloodhound	2.442200e-01	True	flat-coated_retriever	1.738100e-01	True
29	666411507551481857	https://pbs.twimg.com/media/CT-RugiWIAELEaq.jpg	1	coho	0.404640	False	barracouta	2.714850e-01	False	gar	1.899450e-01	False
30	666418789513326592	https://pbs.twimg.com/media/CT-YWb7U8AA7QnN.jpg	1	toy_terrier	0.149680	True	papillon	1.482580e-01	True	Chihuahua	1.428600e-01	True
31	666421158376562688	https://pbs.twimg.com/media/CT-aggCXAAIMfT3.jpg	1	Blenheim_spaniel	0.906777	True	cocker_spaniel	9.034640e-02	True	Shih-Tzu	1.116870e-03	True
32	666428276349472768	https://pbs.twimg.com/media/CT-g-0DUwAEQdSn.jpg	1	Pembroke	0.371361	True	chow	2.493940e-01	True	Pomeranian	2.418780e-01	True
33	666430724426358785	https://pbs.twimg.com/media/CT-jNYqW4AAPi2M.jpg	1	llama	0.505184	False	Irish_terrier	1.041090e-01	True	dingo	6.207120e-02	False
34	666435652385423360	https://pbs.twimg.com/media/CT-nsTQWEAEkyDn.jpg	1	Chesapeake_Bay_retriever	0.184130	True	chain_saw	5.677530e-02	False	power_drill	3.676340e-02	False
35	666437273139982337	https://pbs.twimg.com/media/CT-pKmRWIAAxUWj.jpg	1	Chihuahua	0.671853	True	beagle	1.246800e-01	True	Saluki	4.409420e-02	True
36	666447344410484738	https://pbs.twimg.com/media/CT-yU5QWwAEjLX5.jpg	1	curly-coated_retriever	0.322084	True	giant_schnauzer	2.879550e-01	True	Labrador_retriever	1.663310e-01	True
37	666454714377183233	https://pbs.twimg.com/media/CT-5Bs-WUAA2JeC.jpg	1	dalmatian	0.278954	True	Labrador_retriever	2.376120e-01	True	Great_Pyrenees	1.711060e-01	True
38	666644823164719104	https://pbs.twimg.com/media/CUBl6IwVAAA9_zT.jpg	1	Ibizan_hound	0.044333	True	Pembroke	4.320930e-02	True	West_Highland_white_terrier	3.890560e-02	True
39	666649482315059201	https://pbs.twimg.com/media/CUBqKnLWwAA5OQB.jpg	1	Border_collie	0.447803	True	English_springer	1.704970e-01	True	collie	1.392060e-01	True

	tweet_id	retweet_count	favorite_count
0	892420643555336193	6969	33696
1	892177421306343426	5272	29223
2	891815181378084864	3464	21977
3	891689557279858688	7191	36788
4	891327558926688256	7717	35183
5	891087950875897856	2586	17749
6	890971913173991426	1647	10331
7	890729181411237888	15679	56664
8	890609185150312448	3604	24425
9	890240255349198849	6068	27849
10	890006608113172480	6111	26945
11	889880896479866881	4141	24488
12	889665388333682689	8302	41880
13	889638837579907072	3700	23582
14	889531135344209921	1874	13312
15	889278841981685760	4419	22023
16	888917238123831296	3742	25532
17	888804989199671297	3513	22390
18	888554962724278272	2863	17252
19	888078434458587136	2885	19100
20	887705289381826560	4516	26527
21	887517139158093824	9810	40587
22	887473957103951883	14952	59977
23	887343217045368832	8773	29489
24	887101392804085760	4969	26897
25	886983233522544640	6287	30241
26	886736880519319552	2616	10453
27	886680336477933568	3720	19654
28	886366144734445568	2613	18489
29	886267009285017600	4	105
30	886258384151887873	5253	24444
31	886054160059072513	93	0
32	885984800019947520	5589	28499
33	885528943205470208	5306	31482
34	885518971528720385	3097	17938
35	885311592912609280	15365	0
36	885167619883638784	3652	19190
37	884925521741709313	14943	67014
38	884876753390489601	4639	24299
39	884562892145688576	3896	21243

Project: Wrangling and Analyze Data¶

By: Timothy Yaji¶

Import required packages¶

Data Gathering¶

Assessing Data¶

Summary Issues in the Datasets¶

Quality issues¶

Tidiness issues¶

Cleaning Data¶

Issue #1:¶

Define:¶

Code¶

Test:¶

Issue #2:¶

Define¶

Code¶

Test¶

Issue #3:¶

Define:¶

Code:¶

Test¶

Issue #4:¶

Define:¶

Code¶

Test:¶

Code:¶

Test:¶

Issue #5:¶

Define:¶

Code¶

Test:¶

Issue #6:¶

Define:¶

Code:¶

Test:¶

Issue #7:¶

Define¶

Code¶

Test¶

Issue#8:¶

define:¶

Code:¶

Test:¶

code:¶

Test:¶

Issue #9:¶

Define:¶

Code¶

Test:¶

b. Clean Tidiness Issues¶

Issue #1:¶

Define:¶

Code¶

Test:¶

Issue #2:¶

Define:¶

Code¶

Test:¶

Test¶

Code:¶

Test¶

Storing Data¶

Analyzing and Visualizing Data¶

Insights:¶

Insight #1: What are the top 10 dog breed interms of most likes?¶

visualual for insight #1:¶

insight #2: What dog stage has the most likes and retweets¶

Insight #3: Is there any relationship between likes and retweets?¶

Visualization of insight #3¶